# C:/Python27/TextResuming/main.py
# This is main file
#
from  __future__ import division
from tempfile import TemporaryFile
from xlwt import Workbook
import model.text as text
import util.is_parser as is_parser
import textResuming
import util.preprocess as preprocess
import pickle
import unicodedata
import head_chunker as hc
import word_weighting as ww

def debug(str):
        print "===", str,"==="

def main1():
    try:
        fileInput = 'test/news1'
        fileOutput = fileInput + '_out' + '.txt'
        fileTest = open(fileInput + '.html')
        test = fileTest.readline()
        #test = util.parser.readYAML('test3.yml')
        newText = textResuming.createText(test)
        newText.printText()

        fileNew = open(fileOutput, 'w')
        newText.writeText(fileNew)
        fileNew.close()
        fileTest.close()
    except Exception, e:
        print 'something error', e

def main2():
    print 'test2'
    fileInputs = ['test/10News/news1', 'test/10News/news2', 'test/10News/news3', 'test/10News/news4', 'test/10News/news5', 'test/10News/news6', 'test/10News/news7', 'test/10News/news8', 'test/10News/news9','test/10News/news10']

    book = Workbook()
    for fileInput in fileInputs:
        try:
                #read file
                #fileInput = 'test/10News/news10' #path of file input
                test = is_parser.readTitleLinkNews(fileInput + '.txt')
                sheetName = fileInput.split('/')[-1]
                print "sheetName", sheetName

                #stemming title and content
                # titleStemmed = preprocess.stemSentence(preprocess.normalizeString(preprocess.removePunctuation(test[0])))
                # textStemmed = preprocess.stemListOfSentence(test[2])

                #without stemming
                titleStemmed = preprocess.normalizeString(preprocess.removePunctuation(test[0]))
                textStemmed = test[2]

                #save file
                # fileOutput = fileInput + '_out1' + '.txt'
                # fileNew = open(fileOutput, 'w')
                # fileNew.writelines(titleStemmed)
                # fileNew.writelines('\n' + test[1])
                # fileNew.writelines(' '.join(textStemmed)) #using join for separate sentences

                #create news object
                titleNews = text.title(titleStemmed)

                source = test[1]
                contentNews = ' '.join(textStemmed)
                listSentence = is_parser.parseText2Sentences(contentNews)

                newNews = textResuming.createNews(0, titleNews, source, listSentence)
                
                #init excel file
                activeSheet = book.add_sheet(sheetName)
                activeSheet.write(0,0,'Title : ' + preprocess.normalizeString(test[0]))
                activeSheet.write(1,0,'Stemmed Title : ' + titleStemmed)

                # Write table header
                activeSheet.write(5, 0, 'No')
                activeSheet.write(5, 1, 'Sentence')
                activeSheet.write(5, 2, 'Mutual Word')
                activeSheet.write(5, 3, 'Num of Mutual Word')
                activeSheet.write(5, 4, 'Kedekatan kata')

                #TitleWord sector
                # numTitleWord = newNews.getNumTitleWordInContent()
                # print 'Number of TitleWord found in content :', numTitleWord
                # setTitleWord = newNews.getTitleWordInContent()
                # print setTitleWord

                # activeSheet.write(2,0,'TitleWord found in content : ' + ', '.join(setTitleWord))
                # activeSheet.write(3,0,'Number TitleWord found in content : ' + str(numTitleWord))

                # i = 0
                # while (i < newNews.getNumSentences()):
                #     numOfMutualWord = newNews.getNumTitleWordInSentence(i)
                #     mutualWord = newNews.getSetOfTitleWordInSentence(i)

                #     # print 'sent_idx:', i, 'num of TitleWord:', numOfMutualWord, 'fraction:', (numOfMutualWord / numTitleWord), mutualWord

                #     #write in excel file
                #     activeSheet.write(i+6, 0, i)
                #     activeSheet.write(i+6, 1, unicodedata.normalize('NFKD', newNews.getSentence(i).text.decode('latin-1')).encode('ascii', 'ignore'))
                #     activeSheet.write(i+6, 2, ', '.join(mutualWord))
                #     activeSheet.write(i+6, 3, str(numOfMutualWord))
                #     activeSheet.write(i+6, 4, str(numOfMutualWord/numTitleWord))
                #     i += 1
                

                #TitlePhrase Sector
                numTitlePhrase = newNews.getNumTitlePhraseInContent()
                print 'Number of TitleWord found in content :', numTitlePhrase

                setTitlePhrase = newNews.getTitlePhraseInContent()
                print setTitlePhrase

                activeSheet.write(2,0,'TitlePhrase found in content : ' + ', '.join(setTitlePhrase))
                activeSheet.write(3,0,'Number TitlePhrase found in content : ' + str(numTitlePhrase))

                if numTitlePhrase == 0:
                        numTitlePhrase += 1

                i = 0
                while (i < newNews.getNumSentences()):
                    numOfMutualPhrase = newNews.getNumTitlePhraseInSentence(i)
                    mutualPhrase = newNews.getSetOfTitlePhraseInSentence(i)

                    # print 'sent_idx:', i, 'num of TitleWord:', numOfMutualWord, 'fraction:', (numOfMutualWord / numTitleWord), mutualWord

                    #write in excel file
                    activeSheet.write(i+6, 0, i)
                    activeSheet.write(i+6, 1, unicodedata.normalize('NFKD', newNews.getSentence(i).text.decode('latin-1')).encode('ascii', 'ignore'))
                    activeSheet.write(i+6, 2, ', '.join(mutualPhrase))
                    activeSheet.write(i+6, 3, str(numOfMutualPhrase))
                    activeSheet.write(i+6, 4, str(numOfMutualPhrase/numTitlePhrase))
                    i += 1


                #TitlePhrase
                # print 'News title : ', newNews.getTitle().text
                # print 'TitlePhrase : ', newNews.getSetOfTitlePhrase()

                # numTitlePhrase = newNews.getNumTitlePhraseInContent()
                # print 'Number of TitlePhrase found in content :', numTitlePhrase

                # print newNews.getTitlePhraseInContent()
                # print newNews.getSetOfTitlePhrase()

                # i = 0
                # while (i < newNews.getNumSentences()):
                #         print 'idx', i, newNews.getSetOfPhraseSentence(i)
                #         a = newNews.getNumTitlePhraseInSentence(i)
                #         print 'num Of TitlePhrase: ', a, 'fraction :', (a/numTitlePhrase)
                #         i += 1

                #close stream
                #fileNew.close()

                book.save('result phrase wout stemming.xls')
                book.save(TemporaryFile())

                print 'fin', fileInput
                
        except Exception, e:
                print "main 2 error", e

def main3():
        debug('main3')
        try:
                fileInputs = ['test/10News/news1', 'test/10News/news2', 'test/10News/news3', 'test/10News/news4', 'test/10News/news5', 'test/10News/news6', 'test/10News/news7', 'test/10News/news8', 'test/10News/news9','test/10News/news10']
                test = is_parser.readTitleLinkNews(fileInputs[0] + '.txt')

                #without stemming
                titleStemmed = preprocess.normalizeString(preprocess.removePunctuation(test[0]))
                textStemmed = test[2]

                #create news object
                titleNews = text.title(titleStemmed)

                source = test[1]
                contentNews = ' '.join(textStemmed)
                listSentence = is_parser.parseText2Sentences(contentNews)

                newNews = textResuming.createNews(0, titleNews, source, listSentence)

                dictNews = ww.getDictionary(newNews)

                print dictNews
                
        except Exception, e:
                print 'Exception main3', e
                raise e


if __name__ == '__main__':
    main3();
